{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "cff731c0",
   "metadata": {},
   "source": [
    "## Elasticsearch Indexing  \n",
    "\n",
    "This notebook shows you how to create a Elasticsearch connection, how to create a index, and then how to index your data."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0cb60ea",
   "metadata": {},
   "source": [
    "### 1. Install Elasticsearch library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b5a9c5a5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: https://abhilasha.mangal%40ibm.com:****@na.artifactory.swg-devops.com/artifactory/api/pypi/wcp-nlp-pypi-virtual/simple\n",
      "Requirement already satisfied: elasticsearch===7.10.1 in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (7.10.1)\n",
      "Requirement already satisfied: urllib3<2,>=1.21.1 in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from elasticsearch===7.10.1) (1.26.7)\n",
      "Requirement already satisfied: certifi in /Library/Frameworks/Python.framework/Versions/3.9/lib/python3.9/site-packages (from elasticsearch===7.10.1) (2022.6.15)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m23.1.2\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip3 install elasticsearch===7.10.1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f134b80",
   "metadata": {},
   "source": [
    "<span style=\"color:blueviolet\">Step 1. Provide Elastic user name and password</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "190bc832",
   "metadata": {},
   "outputs": [],
   "source": [
    "username = 'Add user name'\n",
    "password = 'Add password here'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0fdbf544",
   "metadata": {},
   "source": [
    "### 2. Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "0dd78994",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tika\n",
    "tika.initVM()\n",
    "from tika import parser  \n",
    "import re\n",
    "from datetime import date\n",
    "import pandas as pd\n",
    "import json\n",
    "from datetime import datetime\n",
    "import requests\n",
    "import PyPDF2\n",
    "\n",
    "from pathlib import Path\n",
    "from dateutil.parser import parse"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "01be0cf9",
   "metadata": {},
   "source": [
    "### 3. Create Elasticsearch connection\n",
    "<span style=\"color:blueviolet\">Step 2. Provide user name and password and Elasticsearch url for connection</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "5bf0e32d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from elasticsearch import Elasticsearch\n",
    "from elasticsearch.exceptions import RequestError\n",
    "\n",
    "# Create an instance of Elasticsearch with TLS options\n",
    "es_client = Elasticsearch(\n",
    "    'https://<username>:<password>@3d862675-f715-499c-b9e4-ffba4d8321a0.2adb0220806343e3ae11df79c89b377f.databases.appdomain.cloud:32062',\n",
    "    ca_certs='/Users/abhilashamangal/Downloads/5cb6eb86-ae1c-11e9-99c9-6a007ab2fc0b'\n",
    ")\n",
    "info = es_client.info()\n",
    "print(info)\n",
    "\n",
    "index_name = 'superknowa'\n",
    "index_mapping = {\n",
    "    \"mappings\": {\n",
    "        \"properties\": {\n",
    "            \"id\": {\"type\": \"text\"},\n",
    "            \"published_source\": {\"type\": \"text\"},\n",
    "            \"content\": {\"type\": \"text\"},\n",
    "            \"url\": {\"type\": \"text\"},\n",
    "            \"keywords\": {\"type\": \"text\"},\n",
    "            \"categories\": {\"type\": \"text\"},\n",
    "            \"publish_date\": {\"type\": \"date\"},\n",
    "            \"last_update_date\": {\"type\": \"date\"},\n",
    "            \"indexing_date\": {\"type\": \"date\"}\n",
    "            \n",
    "        }\n",
    "    }\n",
    "}\n",
    "try:\n",
    "    es_client.indices.create(index=index_name)\n",
    "    print(f\"Index '{index_name}' created successfully.\")\n",
    "except RequestError as e:\n",
    "    if e.error == 'resource_already_exists_exception':\n",
    "        print(f\"Index '{index_name}' already exists.\")\n",
    "    else:\n",
    "        print(f\"An error occurred while creating index '{index_name}': {e}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22103582",
   "metadata": {},
   "source": [
    "<span style=\"color:blueviolet\">Step 3. Cheking Elastic Search Configration</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "47e893b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'name': 'm-2.3d862675-f715-499c-b9e4-ffba4d8321a0.d5c42fad68fd498ba08f6af6107b71cd.2adb0220806343e3ae11df79c89b377f.databases.appdomain.cloud', 'cluster_name': '3d862675-f715-499c-b9e4-ffba4d8321a0', 'cluster_uuid': 'UsM9ak-LRYajVAwt5yeQxw', 'version': {'number': '7.10.2', 'build_flavor': 'oss', 'build_type': 'tar', 'build_hash': '747e1cc71def077253878a59143c1f785afa92b9', 'build_date': '2021-01-13T00:42:12.435326Z', 'build_snapshot': False, 'lucene_version': '8.7.0', 'minimum_wire_compatibility_version': '6.8.0', 'minimum_index_compatibility_version': '6.0.0-beta1'}, 'tagline': 'You Know, for Search'}\n"
     ]
    }
   ],
   "source": [
    "info = es_client.info()\n",
    "print(info)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a4324b2",
   "metadata": {},
   "source": [
    "<span style=\"color:blueviolet\">Step 4. Providing Elastic Search index name and index mapping</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "70e903c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "index_name = 'superknowa'\n",
    "index_mapping = {\n",
    "    \"mappings\": {\n",
    "        \"properties\": {\n",
    "            \"id\": {\"type\": \"text\"},\n",
    "            \"published_source\": {\"type\": \"text\"},\n",
    "            \"content\": {\"type\": \"text\"},\n",
    "            \"url\": {\"type\": \"text\"},\n",
    "            \"keywords\": {\"type\": \"text\"},\n",
    "            \"categories\": {\"type\": \"text\"},\n",
    "            \"publish_date\": {\"type\": \"date\"},\n",
    "            \"last_update_date\": {\"type\": \"date\"},\n",
    "            \"indexing_date\": {\"type\": \"date\"}\n",
    "            \n",
    "        }\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae08b903",
   "metadata": {},
   "source": [
    "<span style=\"color:blueviolet\">Step 5. Creating Elastic Search index.</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "9849f9b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index 'superknowa' already exists.\n"
     ]
    }
   ],
   "source": [
    "# Test the connection and create an index\n",
    "try:\n",
    "    es_client.indices.create(index=index_name)\n",
    "    print(f\"Index '{index_name}' created successfully.\")\n",
    "except RequestError as e:\n",
    "    if e.error == 'resource_already_exists_exception':\n",
    "        print(f\"Index '{index_name}' already exists.\")\n",
    "    else:\n",
    "        print(f\"An error occurred while creating index '{index_name}': {e}\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "734314cd",
   "metadata": {},
   "source": [
    "<span style=\"color:blueviolet\">Step 6. Processing data</span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "626a5f64",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Today's date: 2023-07-11\n"
     ]
    }
   ],
   "source": [
    "## Extracting data\n",
    "today = date.today()\n",
    "print(\"Today's date:\", today)\n",
    "\n",
    "def get_all_files(folder_name):\n",
    "    # Change the directory\n",
    "    os.chdir(folder_name)\n",
    "    # iterate through all file\n",
    "    file_path_list =[]\n",
    "    for file in os.listdir():\n",
    "        if \".txt\" in file:\n",
    "            file_path = f\"{folder_name}/{file}\"\n",
    "            file_path_list.append(file_path)\n",
    "    return file_path_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5e202bbb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def pre_processingtext(text_data):\n",
    "    replaced = re.sub(\"</?p[^>]*>\", \"\", text_data)\n",
    "    replaced = re.sub(\"</?a[^>]*>\", \"\", replaced)\n",
    "    replaced = re.sub(\"</?h*[^>]*>\", \"\", replaced)\n",
    "    replaced = re.sub(\"</?em*[^>]*>\", \"\", replaced)\n",
    "    replaced = re.sub(\"</?img*[^>]*>\", \"\", replaced)\n",
    "    replaced = re.sub(\"&amp;\", \"\", replaced)\n",
    "    replaced = re.sub(\"id=*>;\", \"\", replaced)\n",
    "    return replaced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "246a5d47",
   "metadata": {},
   "outputs": [],
   "source": [
    "def readdata_frompdf(file_name):\n",
    "    content=''\n",
    "    try:\n",
    "        pdfFileObj = open(file_name, 'rb')\n",
    "        pdfReader = PyPDF2.PdfReader(pdfFileObj)\n",
    "        for i in range(len(pdfReader.pages)):\n",
    "            pageObj = pdfReader.pages[i]\n",
    "            content =content+\" \"+pageObj.extract_text()\n",
    "\n",
    "        pdfFileObj.close()\n",
    "        return content\n",
    "    except:\n",
    "         print(\"file is empty\")\n",
    "         return content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a0468990",
   "metadata": {},
   "outputs": [],
   "source": [
    "### Developer data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "097c837b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Directory containing the documents you can change these dir from your directories \n",
    "ibm_docs_dir = 'Scraper/scrape_data/ibm_developer_metadata/'\n",
    "white_paper_docs_dir ='Scraper/scrape_data/white_paper_metadata/data-2/'\n",
    "redbooks_data_dir ='Scraper/scrape_data/redbooks_data/'\n",
    "ibm_cloud_docs_dir ='Scraper/scrape_data/ibm_cloud_docs_process_metdata_new/'\n",
    "ibm_cloud_metadta_file ='Scraper/scrape_data/ibm_cloud_docs_metadata5.txt'\n",
    "ibm_medium_blog ='Scraper/scrape_data/Medium/text'\n",
    "ibm_medium_blog_csv ='Scraper/scrape_data/Medium/csv'"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6faf538a",
   "metadata": {},
   "source": [
    "## 1. IBM Developer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f9e8879a",
   "metadata": {},
   "outputs": [],
   "source": [
    "def index_ibm_developerdata(file_path,esdocs):\n",
    "    with open(file_path, 'r', encoding=\"latin1\") as file:\n",
    "        if \".txt\" in file_path:\n",
    "            content = file.read()\n",
    "            print(len(content), file_path)\n",
    "            content_value = content.split(\"content:\")\n",
    "            content = pre_processingtext(content_value[1])\n",
    "            categories_val = content_value[0].split(\"categories:\")\n",
    "            categories = categories_val[1]\n",
    "            sub_title =  categories_val[0].split(\"sub_title:\")\n",
    "            title_val =  sub_title[0].split(\"title:\")\n",
    "            title = title_val[1]\n",
    "            pd_val =  title_val[0].split(\"publish_date:\")\n",
    "            publish_date = pd_val[1]\n",
    "            ld_val =  pd_val[0].split(\"updated_date:\")\n",
    "            updated_date = ld_val[1]\n",
    "            print(\"values ---\",len(ld_val))\n",
    "            urls =  ld_val[0].split(\"URL:\")\n",
    "            print(len(urls))\n",
    "           \n",
    "            url = \"https://developer.ibm.com/blogs/\"+urls[1]\n",
    "\n",
    "            indexing_date = today\n",
    "            source = \"IBM Developer\"\n",
    "            data = \"{'id' : '\"+str(title)+\"', 'published_source' : '\"+source+\"', 'publish_date' : '\"+str(publish_date)+\"','last_update_date' : '\"+str(updated_date)+\"','indexing_data' : '\"+str(indexing_date)+\"', 'url' : '\"+url+\"','content' : '\"+str(content)+\"','keywords' : '\"+str(sub_title)+\"','categories' : '\"+str(categories)+\"'}\"\n",
    "           \n",
    "            publish_date = publish_date.replace(\"\\n\",\"\").strip()\n",
    "            updated_date = updated_date.replace(\"\\n\",\"\").strip()\n",
    "            print(publish_date)\n",
    "            print(\"update_date \",updated_date)\n",
    "            publish_date_obj = datetime.strptime(publish_date,\"%Y-%m-%dT%H:%M:%S\")\n",
    "            publish_date = publish_date_obj.date()\n",
    "\n",
    "            updated_date_obj = datetime.strptime(updated_date,\"%Y-%m-%dT%H:%M:%S\")\n",
    "            updated_date = updated_date_obj.date()\n",
    "\n",
    "        \n",
    "            esdocs.append({\n",
    "            \"id\": \"\"+title+\"\",\n",
    "            \"published_source\": \"\"+source+\"\",\n",
    "            \"publish_date\": \"\"+str(publish_date)+\"\",\n",
    "            \"last_update_date\": \"\"+str(updated_date)+\"\",\n",
    "            \"indexing_date\": \"\"+str(indexing_date)+\"\",\n",
    "            \"content\": \"\"+content+\"\",\n",
    "            \"url\": \"\"+url+\"\",\n",
    "            \"keywords\": \"\"+str(sub_title)+\"\",\n",
    "            \"categories\": \"\"+str(categories)+\"\",\n",
    "        })\n",
    "    return esdocs"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7cd7f093",
   "metadata": {},
   "source": [
    "## 2. Redbooks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae1c38e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "### Red Books data\n",
    "esdocs =[]\n",
    "i=0\n",
    "for filename in os.listdir(redbooks_data_dir):\n",
    "        print(\"processing i---\",i)\n",
    "        file_path = os.path.join(redbooks_data_dir, filename)\n",
    "        if i > 1199:\n",
    "            with open(file_path, 'r', encoding=\"latin1\") as file:\n",
    "                extracted_text=''\n",
    "                if \".json\" in file_path:\n",
    "                    content = file.read()\n",
    "                    source =\"Redbooks\"\n",
    "                    content_list = content.split(\"\\n\")\n",
    "                    publish_date = content_list[2].replace(\"publish_date: \",\"\")\n",
    "                    updated_date = content_list[1].replace(\"updated_date: \",\"\")\n",
    "                    if publish_date != '':\n",
    "                        publish_date_obj = datetime.strptime(publish_date,\"%Y-%m-%d\")\n",
    "                        publish_date = publish_date_obj.date()\n",
    "                    if updated_date != '':\n",
    "                        updated_date_obj = datetime.strptime(updated_date,\"%Y-%m-%d\")\n",
    "                        updated_date = updated_date_obj.date()\n",
    "                    if updated_date is None:\n",
    "                        updated_date = publish_date\n",
    "                    if publish_date is None:\n",
    "                        updated_date =today\n",
    "                        publish_date =today\n",
    "                    indexing_date = today\n",
    "                    url = content_list[0].replace(\"URL: \",\"\")\n",
    "                    file_name = content_list[3].replace(\"file_name: \",\"\")\n",
    "                    print(file_name)\n",
    "                    file_path= redbooks_data_dir+file_name\n",
    "                    extracted_text = readdata_frompdf(file_path)\n",
    "                    if extracted_text is not None:\n",
    "                                document ={\n",
    "                                \"id\": \"\"+file_name.replace(\".pdf\",\"\")+\"\",\n",
    "                                \"published_source\": \"\"+source+\"\",\n",
    "                                \"publish_date\": \"\"+str(publish_date)+\"\",\n",
    "                                \"last_update_date\": \"\"+str(indexing_date)+\"\",\n",
    "                                \"indexing_date\": \"\"+str(indexing_date)+\"\",\n",
    "                                \"content\": \"\"+extracted_text+\"\",\n",
    "                                \"url\": \"\"+url+\"\",\n",
    "                                \"keywords\": \"\",\n",
    "                                \"categories\": \"\",\n",
    "                        }\n",
    "                    response = es_client.index(index=index_name, body=document)\n",
    "                    print(response)\n",
    "        i=i+1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "837f6e47",
   "metadata": {},
   "source": [
    "## 3. White Paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "429cdd34",
   "metadata": {},
   "outputs": [],
   "source": [
    "### White Paper Indexing \n",
    "esdocs =[]\n",
    "def indexwhitepaerdata(esdocs):\n",
    "    i =0\n",
    "    for filename in os.listdir(white_paper_docs_dir):\n",
    "        print(\"processing i---\",i)\n",
    "        file_path = os.path.join(white_paper_docs_dir, filename)\n",
    "        \n",
    "        # Read the contents of the file\n",
    "        headers = {\n",
    "                \"X-Tika-OCRLanguage\": \"eng\",\n",
    "                \"X-Tika-OCRTimeout\": \"300\"\n",
    "            }\n",
    "        with open(file_path, 'r', encoding=\"latin1\") as file:\n",
    "            extracted_text=''\n",
    "            if \".json\" in file_path:\n",
    "                content = file.read()\n",
    "                json_object = json.loads(content)\n",
    "                #print(json_object)\n",
    "                source =\"IBM White Paper\"\n",
    "                publish_date = json_object['publish_date']\n",
    "                if publish_date is not None:\n",
    "                    dt = parse(publish_date)\n",
    "                    publish_date=today\n",
    "                else:\n",
    "                    publish_date = today\n",
    "                updated_date = publish_date\n",
    "                indexing_date = today\n",
    "                url = json_object['url']\n",
    "                attachment = json_object['attachment']\n",
    "                title = json_object['title']\n",
    "                j=1\n",
    "                if len(attachment) >1:\n",
    "                    for att in attachment:\n",
    "                        print(\"Value\",att)     \n",
    "                        #file_name = att.replace(\"https://www.ibm.com/support/pages/system/files/inline-files/\",\"\")\n",
    "                        file_name = os.path.basename(att)\n",
    "                        print(file_name)\n",
    "                        file_path = white_paper_docs_dir+file_name\n",
    "                        path = Path(file_path)\n",
    "                        if path.is_file():\n",
    "                            #parsed_content = parser.from_file(file_path,requestOptions={'headers': headers, 'timeout': 500})\n",
    "                            # Get the extracted text\n",
    "                            #extracted_text = parsed_content['content']\n",
    "                            extracted_text =readdata_frompdf(file_path)\n",
    "                        else:\n",
    "                            print(\"file_name\",file_name)\n",
    "                            try:\n",
    "                                downloadFile(att,white_paper_docs_dir+\"/\"+file_name)\n",
    "                                if path.is_file():\n",
    "                                    #parsed_content = parser.from_file(file_path,requestOptions={'headers': headers, 'timeout': 500})\n",
    "                                    # Get the extracted text\n",
    "                                    #extracted_text = parsed_content['content']\n",
    "                                    extracted_text =readdata_frompdf(file_path)\n",
    "                            except:\n",
    "                                print(\"file in not available\")\n",
    "                                break\n",
    "\n",
    "                        title = title+\"_\"+str(i)\n",
    "                        if extracted_text is not None:\n",
    "                            solrdocs.append({\n",
    "                    \"id\": \"\"+title+\"\",\n",
    "                    \"published_source\": \"\"+source+\"\",\n",
    "                    \"publish_date\": \"\"+str(indexing_date)+\"\",\n",
    "                    \"last_update_date\": \"\"+str(updated_date)+\"\",\n",
    "                    \"indexing_date\": \"\"+str(indexing_date)+\"\",\n",
    "                    \"content\": \"\"+extracted_text+\"\",\n",
    "                    \"url\": \"\"+url+\"\",\n",
    "                    \"keywords\": \"\",\n",
    "                    \"categories\": \"\",\n",
    "                })\n",
    "                            \n",
    "                else:\n",
    "                    if len(attachment) ==1:\n",
    "                        file_name = os.path.basename(attachment[0])\n",
    "                        print(file_name)\n",
    "                        file_path = white_paper_docs_dir+file_name\n",
    "                        path = Path(file_path)\n",
    "                        if path.is_file():\n",
    "                            #parsed_content = parser.from_file(file_path,requestOptions={'headers': headers, 'timeout': 500})\n",
    "                            #extracted_text = parsed_content['content']\n",
    "                            extracted_text = readdata_frompdf(file_path)\n",
    "                        if extracted_text is not None:\n",
    "                            solrdocs.append({\n",
    "                        \"id\": \"\"+title+\"\",\n",
    "                        \"published_source\": \"\"+source+\"\",\n",
    "                        \"publish_date\": \"\"+str(indexing_date)+\"\",\n",
    "                        \"last_update_date\": \"\"+str(updated_date)+\"\",\n",
    "                        \"indexing_date\": \"\"+str(indexing_date)+\"\",\n",
    "                        \"content\": \"\"+extracted_text+\"\",\n",
    "                        \"url\": \"\"+url+\"\",\n",
    "                        \"keywords\": \"\",\n",
    "                        \"categories\": \"\",\n",
    "                })\n",
    "    return esdocs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9abb9505",
   "metadata": {},
   "outputs": [],
   "source": [
    "esdocs =indexwhitepaerdata(esdocs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "92c1e0d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "496"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(esdocs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68763420",
   "metadata": {},
   "outputs": [],
   "source": [
    "i =0\n",
    "for document in esdocs:\n",
    "    # Add the document to the index\n",
    "    print(\"index data----\",i)\n",
    "    if i > 226:\n",
    "        response = es_client.index(index=index_name, body=document)\n",
    "        print(response)\n",
    "    i=i+1\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9b4db772",
   "metadata": {},
   "outputs": [],
   "source": [
    "### Index ibm developer data \n",
    "esdocs =[]\n",
    "file_path_list = get_all_files(ibm_docs_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "26e07959",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1472\n"
     ]
    }
   ],
   "source": [
    "print(len(file_path_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "b00a6406",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1472\n"
     ]
    }
   ],
   "source": [
    "print(len(esdocs))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26722f66",
   "metadata": {},
   "source": [
    "### Indexing elastic documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e53f3b66",
   "metadata": {},
   "outputs": [],
   "source": [
    "for document in esdocs:\n",
    "    # Add the document to the index\n",
    "    response = es_client.index(index=index_name, body=document)\n",
    "    print(response)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0409daf",
   "metadata": {},
   "source": [
    "## 4. IBM Developer docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32ea518c",
   "metadata": {},
   "outputs": [],
   "source": [
    "esdocs =[]\n",
    "i=0\n",
    "j=0\n",
    "with open(ibm_cloud_metadta_file, 'r', encoding=\"latin1\") as file:\n",
    "        content = file.read()\n",
    "        content_list = content.split(\"\\n\")\n",
    "        print(len(content_list))\n",
    "        for content in content_list:\n",
    "            #try:\n",
    "                print(\"process---\",i)\n",
    "                file_names = content.split(\"file name\")\n",
    "                if len(file_names)>1:\n",
    "                    url_data = file_names[1].split(\"Url\")\n",
    "                    file_name = url_data[0].replace(\"\\\": \",\"\").replace(\",\\\"\",\"\").replace(\"\\\"\",\"\")\n",
    "                    metadata = url_data[1].split(\"metadata\")\n",
    "                    url = metadata[0].replace(\"\\\": \",\"\").replace(\", \\\"\",\"\")\n",
    "                    lastupdated = metadata[1].split(\"lastupdated:\")\n",
    "               \n",
    "                    \n",
    "                    if len(lastupdated) >1:\n",
    "                        last_updated_data = lastupdated[1].split(\"keywords:\")\n",
    "                        if len(last_updated_data) >1:\n",
    "                            keywords = last_updated_data[1].replace(\"\\\"}\",\"\")\n",
    "                        else:\n",
    "                            keywords=\"\"\n",
    "                    else:\n",
    "                        keywords=\"\"\n",
    "                    last_updated = last_updated_data[0].replace(\"\\\"\",\"\").strip().replace(\" subcollection: assistant }\",\"\").split(\" \")[0]\n",
    "                    if len(last_updated) < 6:\n",
    "                        last_updated = today\n",
    "                    else:\n",
    "                        if '/' in last_updated:\n",
    "                            last_updated = last_updated.replace(\"/\",\"-\") \n",
    "                        if 'b' in last_updated:\n",
    "                            last_updated = last_updated.replace(\"b\",\"-\") \n",
    "                            last_updated = last_updated.replace(\"'\",\"-\") \n",
    "                    #last_updated=last_updated.encode(\"ascii\", \"ignore\")\n",
    "\n",
    "\n",
    "                    print(\"filename\",file_name)\n",
    "                    with open(file_name, 'r', encoding=\"latin1\") as file:\n",
    "                        extracted_text=''\n",
    "                        extracted_text = file.read()\n",
    "                        source =\"IBM Developer docs\"\n",
    "                        #print(extracted_text)\n",
    "\n",
    "                    file_name_val = file_name.replace(\"ibm_cloud_docs_process_metdata_new/\",\"\")\n",
    "                    id = file_name_val.replace(\".txt\",\"\")+\"_\"+source+str(i)\n",
    "                    if extracted_text is not None:\n",
    "                                esdocs.append({\n",
    "                                        \"id\": \"\"+file_name_val.replace(\".txt\",\"\")+\"\",\n",
    "                                        \"published_source\": \"\"+source+\"\",\n",
    "                                        \"publish_date\": \"\"+str(last_updated)+\"\",\n",
    "                                        \"last_update_date\": \"\"+str(last_updated)+\"\",\n",
    "                                        \"indexing_date\": \"\"+str(today)+\"\",\n",
    "                                        \"content\": \"\"+extracted_text+\"\",\n",
    "                                        \"url\": \"\"+url+\"\",\n",
    "                                        \"keywords\": \"\"+keywords+\"\",\n",
    "                                        \"categories\": \"\",\n",
    "                                })\n",
    "                i=i+1\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "3cb74dcd",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7740\n"
     ]
    }
   ],
   "source": [
    "print(len(esdocs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "72e9244b",
   "metadata": {},
   "outputs": [],
   "source": [
    "for document in esdocs:\n",
    "    # Add the document to the index\n",
    "    try:\n",
    "        response = es_client.index(index=index_name, body=document)\n",
    "        print(response)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        continue"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "faf77db1",
   "metadata": {},
   "source": [
    "## 5. IBM Medium Blog"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1c9fa52",
   "metadata": {},
   "outputs": [],
   "source": [
    "esdocs =[]\n",
    "i = 0\n",
    "j = 0\n",
    "file_path_list_medium = get_all_files(ibm_medium_blog)\n",
    "print(type(file_path_list_medium))\n",
    "for filename in os.listdir(ibm_medium_blog_csv):\n",
    "    file_path = os.path.join(ibm_medium_blog_csv, filename)\n",
    "    with open(file_path, 'r', encoding=\"utf-8\") as file:\n",
    "            if \".csv\" in file_path:\n",
    "                content = ''\n",
    "                df = pd.read_csv(file_path, sep='\\t')\n",
    "                for ind in df.index:\n",
    "                    title = df['title'][ind]\n",
    "                    sub_title = df['subtitle'][ind]\n",
    "                    url  =  df['story_url'][ind]\n",
    "                    publish_date = today\n",
    "                    updated_date = today\n",
    "                    file_name = str(ibm_medium_blog+\"/\"+title+\".txt\")\n",
    "                    print(file_name)\n",
    "                    if file_name in file_path_list_medium:\n",
    "                        with open(file_name, 'r', encoding=\"utf-8\") as file1:\n",
    "                                content = file1.read()\n",
    "                               \n",
    "                    indexing_date = today\n",
    "                    source = \"Medium\"\n",
    "                    categories=''\n",
    "                    \n",
    "                    #print(content)\n",
    "\n",
    "                    esdocs.append({\n",
    "                        \"id\": \"\"+title+\"\",\n",
    "                        \"published_source\": \"\"+source+\"\",\n",
    "                        \"publish_date\": \"\"+str(publish_date)+\"\",\n",
    "                        \"last_update_date\": \"\"+str(updated_date)+\"\",\n",
    "                        \"indexing_date\": \"\"+str(indexing_date)+\"\",\n",
    "                        \"content\": \"\"+content+\"\",\n",
    "                        \"url\": \"\"+url+\"\",\n",
    "                        \"keywords\": \"\"+str(sub_title)+\"\",\n",
    "                        \"categories\": \"\"+str(categories)+\"\",\n",
    "                    })\n",
    "                    print(len(solrdocs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f0f12a50",
   "metadata": {},
   "outputs": [],
   "source": [
    "for document in esdocs:\n",
    "    # Add the document to the index\n",
    "    response = es_client.index(index=index_name, body=document)\n",
    "    print(response)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f8ed1b1d",
   "metadata": {},
   "source": [
    "## 6. IBM Product Docs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "099d9d48",
   "metadata": {},
   "outputs": [],
   "source": [
    "ibm_product_file_folder= \"/new-scrape_metadata/\"\n",
    "ibm_product_file_folder_inner= \"/new-scrape_metadata/\"\n",
    "corpus_file_folder_output =\"/new-scrape_metadata1/\"\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a7fae0be",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extarct_all_files(corpus_file_folder,corpus_file_folder_output):\n",
    "    os.chdir(corpus_file_folder)\n",
    "    file_path_list =[]\n",
    "    file_path_list_new =[]\n",
    "    for file in os.listdir():\n",
    "        print(file)\n",
    "        # Check whether file is in text format or not\n",
    "        file_path = f\"{corpus_file_folder}/{file}\"\n",
    "        file_name = file\n",
    "        new_file_path = f\"{corpus_file_folder_output}/{file_name}\"\n",
    "        file_path_list.append(file_path)\n",
    "        file_path_list_new.append(new_file_path)\n",
    "    return file_path_list ,file_path_list_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39f31851",
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path_list ,file_path_list_new =extarct_all_files(ibm_product_file_folder,corpus_file_folder_output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8192c8f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def product_clean_text(text_data_array):\n",
    "    final_text =\"\"\n",
    "    skip =False\n",
    "    for text in text_data_array:\n",
    "        if len(text) > 0:\n",
    "            if len(text) < 10 and text[0].isupper():\n",
    "                skip = True\n",
    "            if 'URL' in text:\n",
    "                skip = True\n",
    "            if 'Copyright' in text:\n",
    "                skip = True\n",
    "            if 'Reference' in text:\n",
    "                skip = True\n",
    "            if skip:\n",
    "                skip = False\n",
    "            else:\n",
    "                text = re.sub(r\"http\\S+\", \"\", text)\n",
    "                final_text = final_text+\"\\n\"+text\n",
    "    return final_text\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "3f45273d",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Today's date: 2023-07-25\n"
     ]
    }
   ],
   "source": [
    "from datetime import datetime\n",
    "from datetime import date\n",
    "## Extracting data\n",
    "today = date.today()\n",
    "print(\"Today's date:\", today)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b7db9d4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def index_ibm_product_data(file_path_list):\n",
    "    i = 0\n",
    "    esdocs =[]\n",
    "    for file_name in file_path_list:\n",
    "        if file_name != 'new-scrape_metadata/HTML':\n",
    "            url = \"\"\n",
    "            title = \"\"\n",
    "            last_updated=\"\"\n",
    "            print(\"processing -----\",i)\n",
    "            text_data = read_text_file(file_name)\n",
    "            text_data_array = text_data.split(\"\\n\")\n",
    "            data =text_data_array[0].split(\"Last Updated:\")\n",
    "            if len(data) > 1:\n",
    "                url_data = data[0].split(\" \",1)\n",
    "                last_updated = data[1]\n",
    "                url = url_data[0]\n",
    "                title = url_data[1]\n",
    "            else:\n",
    "                last_updated =\"\"\n",
    "            content = product_clean_text(text_data_array)\n",
    "            indexing_date = today\n",
    "            source = \"IBM Product Doc whole\"\n",
    "            categories=''\n",
    "            if len(last_updated) > 1:\n",
    "                publish_date = last_updated\n",
    "            else:\n",
    "                publish_date = today\n",
    "                last_updated = today\n",
    "\n",
    "            esdocs.append({\n",
    "                    \"id\": \"\"+title+\"\",\n",
    "                    \"published_source\": \"\"+source+\"\",\n",
    "                    \"publish_date\": \"\"+str(indexing_date)+\"\",\n",
    "                    \"last_update_date\": \"\"+str(last_updated)+\"\",\n",
    "                    \"indexing_date\": \"\"+str(indexing_date)+\"\",\n",
    "                    \"content\": \"\"+content+\"\",\n",
    "                    \"url\": \"\"+url+\"\",\n",
    "                    \"keywords\": \"\"+str(title)+\"\",\n",
    "                    \"categories\": \"\"+str(categories)+\"\",\n",
    "                })\n",
    "        i =i+1\n",
    "        \n",
    "    return esdocs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "acb2e2b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "esdocs = index_ibm_product_data(file_path_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0830955a",
   "metadata": {},
   "source": [
    "## 7. IBM Product Docs Inner pages (HTML)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ad771e31",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_extra_lines(data):\n",
    "    data =re.sub(r'\\n\\s*\\n', '\\n', data, flags=re.MULTILINE)\n",
    "    return data\n",
    "\n",
    "def pre_processingtext(text_data):\n",
    "    replaced = re.sub(\"</?p[^>]*>\", \"\", text_data)\n",
    "    replaced = re.sub(\"</?div[^>]*>\", \"\", text_data)\n",
    "    replaced = re.sub(\"</?a[^>]*>\", \"\", replaced)\n",
    "    replaced = re.sub(\"</?h*[^>]*>\", \"\", replaced)\n",
    "    replaced = re.sub(\"</?em*[^>]*>\", \"\", replaced)\n",
    "    replaced = re.sub(\"</?img*[^>]*>\", \"\", replaced)\n",
    "    replaced = re.sub(\"&amp;\", \"\", replaced)\n",
    "    return replaced\n",
    "\n",
    "def pre_processing_html(html_data):\n",
    "    final_data1 = pre_processingtext(html_data)\n",
    "    final_data1 = remove_extra_lines(final_data1)\n",
    "    return final_data1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "253f1ddc",
   "metadata": {},
   "outputs": [],
   "source": [
    "def index_ibm_product_data_inner(file_path_list):\n",
    "    i = 0\n",
    "    esdocs =[]\n",
    "    for file_name in file_path_list:\n",
    "        url = \"\"\n",
    "        title = \"\"\n",
    "        last_updated=\"\"\n",
    "        print(\"processing -----\",i)\n",
    "        text_data = read_text_file(file_name)\n",
    "        text_data_array = text_data.split(\"\\n\")\n",
    "        last_updated = today\n",
    "        url = text_data_array[0].replace(\"URL:\",\"\")\n",
    "        title = file_name.replace(\"new-scrape_metadata/HTML\",\"\").replace(\".html\",\"\")\n",
    "        content = product_clean_text(text_data_array)\n",
    "        content = pre_processing_html(content)\n",
    "        indexing_date = today\n",
    "        source = \"IBM Product Doc Inner\"\n",
    "        categories=''\n",
    "        publish_date = last_updated\n",
    "        esdocs.append({\n",
    "                    \"id\": \"\"+title+\"\",\n",
    "                    \"published_source\": \"\"+source+\"\",\n",
    "                    \"publish_date\": \"\"+str(indexing_date)+\"\",\n",
    "                    \"last_update_date\": \"\"+str(last_updated)+\"\",\n",
    "                    \"indexing_date\": \"\"+str(indexing_date)+\"\",\n",
    "                    \"content\": \"\"+content+\"\",\n",
    "                    \"url\": \"\"+url+\"\",\n",
    "                    \"keywords\": \"\"+str(title)+\"\",\n",
    "                    \"categories\": \"\"+str(categories)+\"\",\n",
    "                })\n",
    "           \n",
    "        i =i+1\n",
    "        \n",
    "    return esdocs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b089d8e4",
   "metadata": {},
   "outputs": [],
   "source": [
    "esdocs = index_ibm_product_data_inner(file_path_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ecc5d281",
   "metadata": {},
   "outputs": [],
   "source": [
    "for document in esdocs:\n",
    "    # Add the document to the index\n",
    "    try:\n",
    "        response = es_client.index(index=index_name, body=document)\n",
    "        print(response)\n",
    "    except Exception as e:\n",
    "        print(e)\n",
    "        continue"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ae76c17",
   "metadata": {},
   "source": [
    "## Delete the index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1e19a1f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Delete the index\n",
    "response = es_client.indices.delete(index=index_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d4704753",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'acknowledged': True}\n"
     ]
    }
   ],
   "source": [
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8556082f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
